# -*- coding: utf-8 -*-
import scrapy

import re
import logging
from copy import copy, deepcopy
from FDC_spider.constants import QD_RM_STATUS_DICT
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class QingdaoSpider(scrapy.Spider):
    name = 'qingdao'
    allowed_domains = ['qdfd.com.cn']
    start_urls = ['https://www.qdfd.com.cn/qdweb/realweb/fh/FhProjectQueryNew.jsp']
    project_li_url_temp = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhProjectQueryNew.jsp?page={}&rows=20&okey=&order='  # GET 项目列表url
    project_url = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhProjectInfo.jsp'  # POST 项目详情url
    building_li_url_temp = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhBuildingList.jsp?preid={}'  # GET 楼栋列表url
    room_li_url_temp = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhHouseStatus.jsp?buildingID={}&startID={}&projectID={}'  # GET 房号列表url
    room_detail_url_temp = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhHouseDetail.jsp?houseID={}'  # GET 房号详情url
    next_page_url_temp = 'https://www.qdfd.com.cn/qdweb/realweb/fh/FhProjectQueryNew.jsp?page={}&rows=20&okey=&order='  # GET 项目列表翻页url

    def parse(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        try:
            page_num = response.meta['page_num']
            total_page_num = response.meta['total_page_num']
        except:
            page_num = 1  # 设置首页默认值设置为1
            try:
                total_page_str = response.xpath("//a[contains(text(),'末页')]/@onclick").extract_first()
                total_page_num = self._get_taotal_page(total_page_str)
            except:
                total_page_num = 411  # 设置总页数初始值，防止获取出错，无法爬取
                logger.error('总页数获取出错')

        # 获取当前页项目分组
        tr_li = response.xpath("//th[contains(text(),'项目名称')]/../following-sibling::tr")
        if tr_li:
            for tr in tr_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = tr.xpath("./td[1]/a/text()").extract_first()
                item_eg['preSalePermit'] = tr.xpath("./td[2]/text()").extract_first()
                item_eg['permitTotalRoomNum'] = tr.xpath("./td[3]/text()").extract_first()
                item_eg['permitTotalArea'] = tr.xpath("./td[4]/text()").extract_first()
                item_eg['districtName'] = tr.xpath("./td[5]/text()").extract_first()

                # 获取项目详情
                project_id_str = tr.xpath("./td[1]/a/@href").extract_first()
                project_id = self._get_project_id(project_id_str)
                if project_id:
                    item_eg['projectId'] = project_id
                    data = dict(projectID=project_id)
                    yield scrapy.FormRequest(
                        self.project_url,
                        formdata=data,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg)),
                        dont_filter=True,
                        priority=5,
                    )
                else:
                    logger.error('{}-{} 项目id获取出错'.format(item_eg['districtName'], item_eg['projectName'], ))
        else:
            logger.error('第{}页 项目列表获取为空'.format(page_num))

        # 翻页
        for i in range(2, total_page_num + 1):
            page_num = i
            yield scrapy.Request(
                self.next_page_url_temp.format(page_num),
                meta=dict(page_num=deepcopy(page_num), total_page_num=deepcopy(total_page_num), ),
            )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['projectAddress'] = response.xpath(
            "//span[contains(text(),'项目地址')]/following-sibling::span/text()").extract_first()
        item_eg['developerName'] = response.xpath(
            "//span[contains(text(),'企业名称')]/following-sibling::span/a/text()").extract_first()
        item_eg['residenceNum'] = response.xpath("//td[text()='住宅套数']/following-sibling::td[1]/text()").extract_first()
        item_eg['residenceArea'] = response.xpath("//td[text()='住宅面积']/following-sibling::td[1]/text()").extract_first()
        item_eg['totalRoomNum'] = response.xpath("//td[text()='总套数']/following-sibling::td[1]/text()").extract_first()
        item_eg['totalArea'] = response.xpath("//td[text()='总面积']/following-sibling::td[1]/text()").extract_first()
        item_eg['saleableResidenceNum'] = response.xpath(
            "//td[contains(text(),'可售住宅套数')]/following-sibling::td[1]/text()").extract_first()
        item_eg['saleableResidenceArea'] = response.xpath(
            "//td[contains(text(),'可售住宅面积')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalSaleableRoomNum'] = response.xpath(
            "//td[contains(text(),'可售总套数')]/following-sibling::td[1]/text()").extract_first()
        item_eg['totalSaleableArea'] = response.xpath(
            "//td[contains(text(),'可售总面积')]/following-sibling::td[1]/text()").extract_first()
        permit_id_str = response.xpath(f"//a[contains(text(),'{item_eg['preSalePermit']}')]/@href").extract_first()
        item_eg['permitId'] = self._get_permit_id(permit_id_str, item_eg)
        item_eg['openingDate'] = response.xpath(
            f"//a[contains(text(),'{item_eg['preSalePermit']}')]/../../td[3]/text()").extract_first()
        item_eg['permitSaleableRoomNum'] = response.xpath(
            f"//a[contains(text(),'{item_eg['preSalePermit']}')]/../../td[8]/text()").extract_first()
        item_eg['permitSaleableArea'] = response.xpath(
            f"//a[contains(text(),'{item_eg['preSalePermit']}')]/../../td[9]/text()").extract_first()
        yield item_eg

        # 获取楼栋列表
        yield scrapy.Request(
            self.building_li_url_temp.format(item_eg['permitId']),
            callback=self.parse_building_li,
            meta=dict(item_eg=deepcopy(item_eg), ),
            priority=6,
        )

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取楼栋列表分组
        tr_li = response.xpath("//table/tr")[1:]  # 首行为标题栏
        if tr_li:
            for tr in tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']
                item_bd['projectId'] = item_eg['projectId']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['permitId'] = item_eg['permitId']
                item_bd['blockName'] = tr.xpath("./td[1]/a/text()").extract_first()
                item_bd['referencePrice'] = tr.xpath("./td[2]/text()").extract_first()
                item_bd['floatingRange'] = tr.xpath("./td[3]/text()").extract_first()
                item_bd['saleableRoomNo'] = tr.xpath("./td[4]/text()").extract_first()
                item_bd['reserveRoomNo'] = tr.xpath("./td[5]/text()").extract_first()
                item_bd['totalRoomNo'] = tr.xpath("./td[6]/text()").extract_first()
                building_id_str = tr.xpath("./td[1]/a/@href").extract_first()
                item_bd['buildingId'], rm_li_url = self._get_building_id_and_room_li_url(building_id_str, item_bd)
                yield item_bd

                # 获取房号列表
                yield scrapy.Request(
                    rm_li_url,
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd), ),
                    priority=7,
                )
        else:
            logger.error(
                '{}  {}-{} 楼栋列表获取为空'.format(response.request.url, item_eg['projectName'], item_eg['preSalePermit']))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        # 获取楼层分组
        tr_li = response.xpath("//*[contains(text(),'室号')]/../following-sibling::tr")
        if tr_li:
            for tr in tr_li:
                floor_num = tr.xpath("./td[1]/text()").extract_first()
                # 获取当前楼层的房号分组
                td_li = tr.xpath("./td[position()>1]")
                if td_li:
                    for td in td_li:
                        item_rm = FdcRoomItem()
                        item_rm['blockName'] = item_bd['blockName']
                        item_rm['buildingId'] = item_bd['buildingId']
                        item_rm['physicsFloor'] = floor_num
                        item_rm['saleStatus'] = QD_RM_STATUS_DICT.get(td.xpath("./@bgcolor").extract_first(), None)
                        text_str = td.xpath(".//text()").extract_first()
                        item_rm['roomNo'] = self._get_room_no(text_str)
                        room_url_str = td.xpath("./a/@href").extract_first()
                        if room_url_str:
                            room_id = self._get_room_id(room_url_str)
                            if room_id:
                                # 获取房号详情
                                yield scrapy.Request(
                                    self.room_detail_url_temp.format(room_id),
                                    callback=self.parse_room_detail,
                                    meta=dict(item_rm=deepcopy(item_rm), ),
                                    priority=8,
                                )
                            else:
                                logger.error(
                                    '{} {}-{}-{}-第{}层-{} 房号id获取失败'.format(response.request.url, item_bd['projectName'],
                                                                          item_bd['preSalePermit'],
                                                                          item_bd['blockName'], floor_num,
                                                                          item_rm['roomNo']))
                        else:
                            item_rm['nominalFloor'] = None
                            item_rm['roomUse'] = None
                            item_rm['forecastBuildArea'] = None
                            item_rm['forecastInnerArea'] = None
                            item_rm['forecastSharedArea'] = None
                            item_rm['unitPrice'] = None
                            yield item_rm
                else:
                    logger.warning('{} {}-{}-{}-第{}层 房号列表获取为空'.format(response.request.url, item_bd['projectName'],
                                                                      item_bd['preSalePermit'], item_bd['blockName'],
                                                                      floor_num))
        else:
            logger.error(
                '{}  {}-{}-{} 楼层分组获取为空'.format(response.request.url, item_bd['projectName'], item_bd['preSalePermit'],
                                               item_bd['blockName']))

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        floor_str = response.xpath("//td[contains(text(),'实际层')]/following::td[1]/text()").extract_first()
        item_rm['nominalFloor'] = self._get_floor_num(floor_str)
        item_rm['roomUse'] = response.xpath("//td[contains(text(),'房屋类型')]/following::td[1]/text()").extract_first()
        item_rm['forecastBuildArea'] = response.xpath(
            "//td[contains(text(),'预测建筑面积')]/following::td[1]/text()").extract_first()
        item_rm['forecastInnerArea'] = response.xpath(
            "//td[contains(text(),'预测套内面积')]/following::td[1]/text()").extract_first()
        item_rm['forecastSharedArea'] = response.xpath(
            "//td[contains(text(),'预测分摊面积')]/following::td[1]/text()").extract_first()
        item_rm['unitPrice'] = response.xpath("//td[contains(text(),'参考价格')]/following::td[1]/text()").extract_first()
        yield item_rm

    @staticmethod
    def _get_taotal_page(_str: str) -> int or None:
        """
        获取总页数
        :param _str: str or None
        :return: int or None
        """
        regex = re.compile(r'\d+')
        try:
            ret = int(regex.findall(_str)[0])
            assert ret > 1
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_project_id(_str: str) -> str or None:
        """
        获取项目id
        :param _str: str or None
        :return: str or None
        """
        regex = re.compile(r'\"(.*)\"')
        try:
            ret = regex.findall(_str)[0]
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_permit_id(_str: str, item) -> str or None:
        """
        获取许可证id
        :param _str: str or None
        :return: str or None
        """
        regex = re.compile(r'\d+')
        try:
            ret = regex.findall(_str)[0]
        except:
            logger.error('{}-{} 许可证id提取出错'.format(item['projectName'], item['preSalePermit']))
        else:
            return ret

    def _get_building_id_and_room_li_url(self, _str: str, item) -> tuple:
        """
        获取楼栋id 和 房号列表url
        :param _str: str or None
        :param item: item_bd
        :return:
        """
        regex = re.compile(r'\d+')
        try:
            ret_li = regex.findall(_str)
            assert len(ret_li) == 3 and ret_li[2] == str(item['projectId'])
            building_id = ret_li[0]
            room_li_url = self.room_li_url_temp.format(ret_li[0], ret_li[1], ret_li[2])
        except:
            logger.error(
                '{}-{}-{} 楼栋id和房号列表url 获取失败'.format(item['projectName'], item['preSalePermit'], item['blockName']))
            return None, None
        else:
            return building_id, room_li_url

    @staticmethod
    def _get_room_no(_str: str) -> str:
        """
        提取房号
        :param _str: str
        :return: str
        """
        regex = re.compile(r'\d+')
        ret_str = _str.replace('\xa0', '')
        try:
            ret = regex.findall(ret_str)[0]
            assert ret
        except:
            return ret_str
        else:
            return ret

    def _get_room_id(self, _str: str) -> str or None:
        """
        获取房号url
        :param _str: str
        :return: str
        """
        regex = re.compile(r"\('(.*)'\)")
        try:
            ret = regex.findall(_str)[0]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_floor_num(_str: str) -> str or None:
        """
        获取楼层(名义层)
        :param _str: str or None
        :return: str or None
        """
        regex = re.compile(r'\d+')
        try:
            ret = regex.findall(_str)[0]
        except:
            return
        else:
            return ret